PC Pro 2005 June (DVD)/DPPRO0605DVD.iso/Install/program files/Borland/BDS/3.0/Demos/Delphi.Net/VCL/Lexer/mwD4NLexer.pas

Pascal/Delphi Source File | 2004-10-22 | 63.7 KB | 2,291 lines

{-----------------------------------------------------------------------------
This Software is placed into Publlic Domain and distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied.
The Initial Developer is Martin Waldenburg
([e-mail]).
Contributor: James Jacobson

This lexer is processing sloppy UTF8,
for copatibility to Delphi up to D7.
Characters below "C0" are handled as
single bytes characters.

00-7F  Single byte character
80-BF  Trailing byte
C0-DF  Leading byte of a two byte character
E0-EF  Leading byte of a three byte character
F0-F7  Leading byte of a four byte character
F8-FB  Illegal (formerly Leading byte of a five byte character)
FC-FD  Illegal (formerly Leading byte of a six byte character)
FE-FF  Illegal

Byte-order marks
EF BB BF         UTF-8
FE FF           UTF-16/UCS-2, little endian
FF FE           UTF-16/UCS-2, big endian
FF FE 00 00     UTF-32/UCS-4, little endian.
00 00 FE FF     UTF-32/UCS-4, big-endian.
-----------------------------------------------------------------------------}
unit mwD4NLexer;

interface

uses
{$IFDEF CLR}
  System.Text,
{$ENDIF}
  SysUtils, Classes, Windows, Contnrs,
  mwDelphiLanguageElements;

const
  UTF8Width: array[#0..#255] of Byte = (
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6);

type
  TmwLexerError = procedure(Sender: TObject; Error: string);

  TmwBinHashItem = class
    HashValue: Integer;
    Key: AnsiString;
  end;

  TmwBinHashItemList = array of TmwBinHashItem;

  TmwBinHash = class(TObject)
  protected
    FList: TmwBinHashItemList;
    UpperMap: array[#0..#255] of AnsiChar;
    Sorted: Boolean;
    fCount: Integer;
    fCapacity: Integer;
    procedure AddSorted(const Item: TmwBinHashItem); overload;
    function CompareValue(const Value1, Value2: Integer): Integer;
    function CompareString(const S1, S2: AnsiString): Boolean;
    procedure Expand;
    function HashOf(const Key: AnsiString): Integer; overload;
    procedure InitHashTable;
    procedure set_Capacity(NewCapacity: Integer);
    procedure Sort;
    property Capacity: Integer read fCapacity write set_Capacity;
  public
    constructor Create;
    procedure Add(const Item: TmwBinHashItem); overload;
    procedure Add(const S: AnsiString); overload;
    procedure AddSorted(const S: AnsiString); overload;
    procedure Clear; virtual;
    procedure Delete(Index: Integer);
    function get_Items(Index: Integer): TmwBinHashItem;
    function IndexOf(S: AnsiString): Integer;
    procedure Insert(Index: Integer; Item: TmwBinHashItem);
    function Remove(S: AnsiString): Integer;
    property Count: Integer read fCount;
    property Items[Index: Integer]: TmwBinHashItem read get_Items; default;
    property List: TmwBinHashItemList read FList;
  end;

  TmwD4NLexer = class;

  TmwLexKeyListItem = class
    S: AnsiString;
    HashValue: Integer;
    ExId: Integer;
    Id: Integer;
  end;

{$IFDEF CLR}
  TmwLexSubKeyList = array of TmwLexKeyListItem;
{$ELSE}
  PmwLexSubKeyList = ^TmwLexSubKeyList;
  TmwLexSubKeyList = array[0..0] of TmwLexKeyListItem;
{$ENDIF}

  TmwLexKeyList = class(TObject)
  private
    Selector: array[0..255] of array of TmwLexKeyListItem;
    function Compare(const aStart: Integer; const S: AnsiString): Boolean;
  protected
    Owner: TmwD4NLexer;
  public
    constructor Create(aOwner: TmwD4NLexer);
    destructor Destroy; override;
    procedure Add(const S: AnsiString; anId, anExId: Integer);
    procedure Clear;
    procedure Hash(const aStart: Integer);
  end;

  TmwDelphiRange = (
    drNormal,
    drAnsiDirective,
    drBorlandDirective,
    drAnsiComment,
    drAssemblerReference,
    drBorlandComment
    );

  TmwAppType = (
    atGUI,
    atConsole
    );

  TbdBooleanDirectives = (
    bdAlign,
    bdAssertions,
    bdBooleval,
    bdDebugInfo,
    bdDenyPackageUnit,
    bdDesignOnly,
    bdObjExportAll,
    bdExtendedSyntax,
    bdHints,
    bdImplicitBuild,
    bdImportedData,
    bdIOChecks,
    bdLocalSymbols,
    bdLongStrings,
    bdOpenStrings,
    bdOptimization,
    bdOverFlowChecks,
    bdRangeChecks,
    bdRealCompatibility,
    bdRunOnly,
    bdStackChecks,
    bdTypeInfo,
    bdDefinitionInfo,
    bdSafeDivide,
    bdTypedAddress,
    bdVarStringChecks,
    bdWarnings,
    bdWeakPackageUnit,
    bdStackFrames,
    bdWriteableConst,
    bdSymbol_Platform,
    bdSymbol_Library,
    bdSymbol_Deprecated,
    bdUnit_Deprecated,
    bdUnit_Library,
    bdUnit_Platform);

  TmwBooleanDirectives = set of TbdBooleanDirectives;

  TmwLexData = class
  public
    ExId: Integer;
    FileName: AnsiString;
    Id: Integer;
    LinePosition: Integer;
    LineNumber: Integer;
{$IFDEF CLR}
    Buf: array of AnsiChar;
{$ELSE}
    Buf: PChar;
{$ENDIF}
    Range: TmwDelphiRange;
    Run: Integer;
    Start: Integer;
    TheEnd: Integer;
    RecordAlignment: Byte;
    BooleanDirectives: TmwBooleanDirectives;
  end;

  TmwD4NLexer = class(TObject)
  private
    function GetLinePosition: Integer;
    function GetRun: Integer;
    function GetStart: Integer;
    function GetColumn: Integer;
    function GetToken: AnsiString;
{$IFDEF CLR}
    function GetTokenWide: string;
{$ELSE}
    function GetTokenWide: WideString;
{$ENDIF}
  protected
    fDescription: AnsiString;
    fExtension: AnsiString;
    fExternalSym: AnsiString;
    fHppEmit: AnsiString;
    fRecordAlignment: Byte;
    fBooleanDirectives: TmwBooleanDirectives;
    fApptype: TmwAppType;
    fImageBase: Integer;
    fResourceReserve: AnsiString;
    fMaxStackSize: Integer;
    fMinStackSize: Integer;
    fMinEnumSize: Integer;
    fNoDefine: AnsiString;
    fNoInclude: AnsiString;
    fResFileName: AnsiString;
    fRcFileName: AnsiString;
    fRegionString: AnsiString;
    fDefinedList: TmwBinHash;
    fDeclaredList: TmwBinHash;
    fDefaultDefines: TStringList;
    fEmitForHpp: Boolean;
    fOnLexerError: TmwLexerError;
    DirectiveStack: TStack;
    KeyList: TmwLexKeyList;
    DirectiveKeyList: TmwLexKeyList;
    UpperMap: array[0..255] of Integer;
    InIdentifiers: array[#0..#255] of Boolean;
    InInternationalIdentifiers: array[#0..#255] of Boolean;
    procedure CreateKeyLists;
    procedure DestroyKeyLists;
    procedure InitTables;
    procedure InitializeKeyTables;
    procedure InitializeDirectives;
    function RetrieveIncludeFileName: AnsiString;
    function RetrieveResourceFileName: AnsiString;
    procedure SetDefaultDefines(Value: TStringList);
    procedure SetDescription(Value: AnsiString);
    procedure SetExtension(Value: AnsiString);
    procedure SetExternalSym(Value: AnsiString);
    procedure SetHppEmit(Value: AnsiString);
    procedure SetImageBase(Value: Integer);
    procedure SetResourceReserve(Value: AnsiString);
    procedure SetMaxStackSize(Value: Integer);
    procedure SetMinStackSize(Value: Integer);
    procedure SetMinEnumSize(Value: Integer);
    procedure SetNoDefine(Value: AnsiString);
    procedure SetNoInclude(Value: AnsiString);
    procedure SetResFileName(Value: AnsiString);
    procedure SetRcFileName(Value: AnsiString);
    procedure SetRegionString(Value: AnsiString);
    procedure IdentifierHandler;
    procedure InternationalIdentifierHandler;
    procedure UTF8IdentifierHandler;
    procedure SpaceHandler;
    procedure LFHandler;
    procedure CRHandler;
    procedure ExclamHandler; // 33
    procedure DoubleQuoteHandler; // 34
    procedure SharpHandler; // 35
    procedure DollarHandler; // 36
    procedure PercentHandler; // 37
    procedure AmpersandHandler; // 38
    procedure ApostropheHandler; // 39
    procedure RoundOpenHandler; // 40
    procedure RoundCloseHandler; // 41
    procedure AsteriskHandler; // 42
    procedure PlusHandler; // 43
    procedure CommaHandler; // 44
    procedure MinusHandler; // 45
    procedure PeriodHandler; // 46
    procedure SlashHandler; // 47
    procedure NumberHandler; // 48 - 57
    procedure ColonHandler; // 58
    procedure SemicolonHandler; // 59
    procedure LessHandler; // 60
    procedure EqualHandler; // 61
    procedure GreaterHandler; // 62
    procedure QuestionHandler; // 63
    procedure AtHandler; // 64
    procedure SquareOpenHandler; // 91
    procedure BackslashHandler; // 92
    procedure SquareCloseHandler; // 93
    procedure AsciiCircumHandler; // 94
    procedure QuoteLeftHandler; // 96
    procedure CurlyOpenHandler; // 123
    procedure BarHandler; // 124
    procedure CurlyCloseHandler; // 125
    procedure AsciiTildeHandler; // 126
    procedure UnknownHandler;

    procedure NextAnsiComment;
    procedure NextAssemblerReference;
    procedure NextBorlandComment;

    procedure AnsiCommentHandler;
    procedure AssemblerReferenceHandler;
    procedure BorlandCommentHandler;
  public
    ExId: Integer;
    FileName: AnsiString;
    Id: Integer;
    fLinePosition: Integer;
    fLineNumber: Integer;
{$IFDEF CLR}
    Buf: array of AnsiChar;
{$ELSE}
    Buf: PChar;
{$ENDIF}
    Range: TmwDelphiRange;
    Run: Integer;
    Start: Integer;
    TheEnd: Integer;
    constructor Create;
    destructor Destroy; override;
    procedure AssignData(Source: TmwLexData);
    procedure AssignToData(Dest: TmwLexData);
    function Equals(const S: AnsiString): Boolean;
    procedure Next;
    procedure NextNoSpace;
    procedure NextToken; virtual;
{$IFDEF CLR}
    procedure SetBuf(const aBuf: array of AnsiChar);
{$ELSE}
    procedure SetBuf(const aBuf: PChar; Len: Integer);
{$ENDIF}
    property Apptype: TmwAppType read fApptype write fApptype;
    property BooleanDirectives: TmwBooleanDirectives read fBooleanDirectives write fBooleanDirectives;
    property Column: Integer read GetColumn;
    property DefaultDefines: TStringList read fDefaultDefines write SetDefaultDefines;
    property DefinedList: TmwBinHash read fDefinedList;
    property DeclaredList: TmwBinHash read fDeclaredList;
    property Description: AnsiString read fDescription write SetDescription;
    property EmitForHpp: Boolean read fEmitForHpp write fEmitForHpp;
    property Extension: AnsiString read fExtension write SetExtension;
    property ExternalSym: AnsiString read fExternalSym write SetExternalSym;
    property HppEmit: AnsiString read fHppEmit write SetHppEmit;
    property ImageBase: Integer read fImageBase write SetImageBase;
    property MaxStackSize: Integer read fMaxStackSize write SetMaxStackSize;
    property MinStackSize: Integer read fMinStackSize write SetMinStackSize;
    property MinEnumSize: Integer read fMinEnumSize write SetMinEnumSize;
    property NoDefine: AnsiString read fNoDefine write SetNoDefine;
    property NoInclude: AnsiString read fNoInclude write SetNoInclude;
    property LineNumber: Integer read fLineNumber;
    property LinePosition: Integer read GetLinePosition;
    property RecordAlignment: Byte read fRecordAlignment write fRecordAlignment;
    property RegionString: AnsiString read fRegionString write SetRegionString;
    property ResourceReserve: AnsiString read fResourceReserve write SetResourceReserve;
    property ResFileName: AnsiString read fResFileName write SetResFileName;
    property RcFileName: AnsiString read fRcFileName write SetRcFileName;
    property RunPos: Integer read GetRun;
    property StartPos: Integer read GetStart;
    property Token: AnsiString read GetToken;
{$IFDEF CLR}
    property WideToken: string read GetTokenWide;
{$ELSE}
    property WideToken: WideString read GetTokenWide;
{$ENDIF}
    property OnLexerError: TmwLexerError read fOnLexerError write fOnLexerError;
  end;

implementation

{ TmwBinHash }

procedure TmwBinHash.Add(const S: AnsiString);
var
  Item: TmwBinHashItem;
begin
  Item := TmwBinHashItem.Create;
  Item.Key := S;
  Add(Item);
end;

procedure TmwBinHash.Add(const Item: TmwBinHashItem);
begin
  if Item <> nil then
  begin
    Item.HashValue := HashOf(Item.Key);
    if fCount < 7000 then
    begin
      AddSorted(Item);
    end else
    begin
      Sorted := False;
      Insert(fCount, Item);
    end;
  end;
end;

procedure TmwBinHash.AddSorted(const S: AnsiString);
var
  Item: TmwBinHashItem;
begin
  Item := TmwBinHashItem.Create;
  Item.Key := S;
  Item.HashValue := HashOf(Item.Key);
  AddSorted(Item);
end;

procedure TmwBinHash.AddSorted(const Item: TmwBinHashItem);
var
  Value, First, Last, Temp: Integer;
  Larger: ByteBool;
begin
  Value := Item.HashValue;
  Larger := False;
  Temp := 0;
  if fCount > 0 then
  begin
    if not sorted then Sort;
    First := 0;
    Last := fCount - 1;
    while First <= Last do
    begin
      Temp := (First + Last) shr 1;
      case CompareValue(Value, TmwBinHashItem(fList[Temp]).HashValue) of
        -1:
          begin
            Last := Temp - 1;
            Larger := False;
          end;
        0:
          begin
            Larger := False;
            break;
          end;
        1:
          begin
            First := Temp + 1;
            Larger := True;
          end;
      end;
    end;
    case Larger of
      True: Insert(Temp + 1, Item);
      False: Insert(Temp, Item);
    end;
  end else Insert(0, Item);
end;

procedure TmwBinHash.Clear;
var
  I: Integer;
begin
  for I := 0 to fCount - 1 do
    fList[I].Free;
  SetLength(fList, 0);
  fCount := 0;
  fCapacity := 0;
  Sorted := True;
end;

function TmwBinHash.CompareString(const S1, S2: AnsiString): Boolean;
var
  I: Integer;
begin
  Result := False;
  if Length(S1) <> Length(S2) then Exit;
  begin
    Result := True;
    for I := Length(S1) downto 1 do
      if S1[I] <> S2[I] then
        if UpperMap[S1[I]] <> UpperMap[S2[I]] then
        begin
          Result := False;
          Exit;
        end;
  end;
end;

function TmwBinHash.CompareValue(const Value1, Value2: Integer): Integer;
begin
  Result := 0;
  if Value1 < Value2 then Result := -1 else
    if Value1 > Value2 then Result := 1;
end;

constructor TmwBinHash.Create;
begin
  inherited Create;
  InitHashTable;
end;

procedure TmwBinHash.Delete(Index: Integer);
begin
  if (Index >= 0) and (Index < fCount) then
  begin
{$IFDEF CLR}
    dec(fCount);
    System.array.Copy(FList, Index + 1, FList, Index,
      fCount - Index);
{$ELSE}
    FList[Index].Free;
    dec(fCount);
    if Index < fCount then
      System.Move(FList[Index + 1], FList[Index],
        (fCount - Index) * SizeOf(TmwBinHashItem));
{$ENDIF}
  end;
end;

procedure TmwBinHash.Expand;
begin
  set_Capacity(fCount + 2 + fCount div 10);
end;

function TmwBinHash.get_Items(Index: Integer): TmwBinHashItem;
begin
  Result := TmwBinHashItem(FList[Index]);
end;

function TmwBinHash.HashOf(const Key: AnsiString): Integer;
{modified version of HashDKC3 posted to  delphi.language.basm}
var
{$IFDEF CLR}
  I: Integer;
{$ELSE}
  P, P2: PChar;
{$ENDIF}
begin
  Result := 0;
{$IFDEF CLR}
  for I := 1 to Length(Key) do
    Result := Result shl 10 - Result shl 5 - Result + Ord(Key[I]);
{$ELSE}
  P := PChar(Key);
  P2 := P + Length(Key);
  while P < P2 do
  begin
    Result := Result shl 10 - Result shl 5 - Result + Ord(P^);
    inc(P);
  end;
{$ENDIF}
end;

function TmwBinHash.IndexOf(S: AnsiString): Integer;
var
  Value, First, Last, Temp: Integer;
begin
  if not sorted then Sort;
  Value := HashOf(s);
  Result := -1;
  First := 0;
  Last := fCount - 1;
  Temp := -1;
  while First <= Last do
  begin
    Temp := (First + Last) shr 1;
    if Value < TmwBinHashItem(fList[Temp]).HashValue then
      Last := Temp - 1
    else
      if Value > TmwBinHashItem(fList[Temp]).HashValue then
        First := Temp + 1
      else Break;
  end;
  if Temp = -1 then Exit;
  repeat
    dec(Temp);
  until (Temp < 0) or (Value <> TmwBinHashItem(fList[Temp]).HashValue);
  inc(Temp);
  while (Temp < fCount) and (Value = TmwBinHashItem(fList[Temp]).HashValue) do
    case CompareString(S, TmwBinHashItem(fList[Temp]).Key) of
      True:
        begin
          Result := Temp;
          Exit;
        end;
      False: inc(Temp);
    end;
end;

procedure TmwBinHash.InitHashTable;
var
  I: AnsiChar;
  C: array[0..1] of AnsiChar;
  S: AnsiString;
begin
  C[1] := #0;
  for I := #0 to #255 do
  begin
    C[0] := I;
    S := AnsiUpperCase(C);
    case S <> '' of
      True: UpperMap[I] := S[1];
      False: UpperMap[I] := I;
    end;
  end;
end;

procedure TmwBinHash.Insert(Index: Integer; Item: TmwBinHashItem);
begin
  if fCount = Capacity then
    Expand;
{$IFDEF CLR}
  if Index < fCount then
    System.array.Copy(FList, Index, FList, Index + 1,
      fCount - Index);
{$ELSE}
  if Index < fCount then
    System.Move(FList[Index], FList[Index + 1],
      (fCount - Index) * SizeOf(TmwBinHashItem));
{$ENDIF}
  FList[Index] := Item;
  inc(fCount);
end;

function TmwBinHash.Remove(S: AnsiString): Integer;
begin
  Result := IndexOf(S);
  if Result >= 0 then
    Delete(Result);
end;

procedure TmwBinHash.set_Capacity(NewCapacity: Integer);
var
  I: Integer;
begin
  if NewCapacity <> Capacity then
  begin
    if NewCapacity < fCount then
      for I := NewCapacity to fCount - 1 do
        fList[I].Free;
    SetLength(fList, NewCapacity);
    FCapacity := NewCapacity;
  end;
end;

{ Based on a non-recursive QuickSort from the SWAG-Archive.
  ( TV Sorting Unit by Brad Williams ) }

procedure TmwBinHash.Sort;
var
  Left, Right, SubArray, SubLeft, SubRight: LongInt;
  Temp, Pivot: TmwBinHashItem;
  Stack: array[1..32] of record First, Last: LongInt;
  end;
begin
  if fCount < 2 then Exit;
  SubArray := 1;
  Stack[SubArray].First := 0;
  Stack[SubArray].Last := fCount - 1;
  repeat
    Left := Stack[SubArray].First;
    Right := Stack[SubArray].Last;
    Dec(SubArray);
    repeat
      SubLeft := Left;
      SubRight := Right;
      Pivot := TmwBinHashItem(fList[(Left + Right) shr 1]);
      repeat
        while CompareValue(TmwBinHashItem(fList[SubLeft]).HashValue, Pivot.HashValue) < 0 do Inc(SubLeft);
        while CompareValue(TmwBinHashItem(fList[SubRight]).HashValue, Pivot.HashValue) > 0 do Dec(SubRight);
        if SubLeft <= SubRight then
        begin
          Temp := TmwBinHashItem(fList[SubLeft]);
          fList[SubLeft] := fList[SubRight];
          fList[SubRight] := Temp;
          Inc(SubLeft);
          Dec(SubRight);
        end;
      until SubLeft > SubRight;
      if SubLeft < Right then
      begin
        Inc(SubArray);
        Stack[SubArray].First := SubLeft;
        Stack[SubArray].Last := Right;
      end;
      Right := SubRight;
    until Left >= Right;
  until SubArray = 0;
  Sorted := True;
end;

{ TmwLexKeyList }

procedure TmwLexKeyList.Add(const S: AnsiString; anId, anExId: Integer);
var
  Key: Integer;
  Item: TmwLexKeyListItem;
{$IFDEF CLR}
  Sub: TmwLexSubKeyList;
{$ELSE}
  Sub: PmwLexSubKeyList;
{$ENDIF}
  Value2, Count, First, Last, Temp: Integer;
  Larger: Boolean;
begin
  case S = '' of
    True: Exit;
    False:
      begin
        Item := TmwLexKeyListItem.Create;
        Item.S := S;
        Item.HashValue := Owner.UpperMap[Ord(S[Length(S)])] shl Length(S);
        Item.Id := anId;
        Item.ExId := anExId;
        Key := Owner.UpperMap[Ord(S[1])];
        Count := Length(Selector[Key]);
{$IFDEF CLR}
        Sub := Selector[Key];
{$ELSE}
        Sub := Pointer(Selector[Key]);
{$ENDIF}
        Larger := False;
        Temp := 0;
        if Count > 0 then
        begin
          First := 0;
          Last := Count - 1;
          while First <= Last do
          begin
            Temp := (First + Last) shr 1;
            Value2 := Sub[Temp].HashValue;
            if Item.HashValue < Value2 then
            begin
              Last := Temp - 1;
              Larger := False;
            end
            else
              if Item.HashValue > Value2 then
              begin
                First := Temp + 1;
                Larger := True;
              end
              else
              begin
                Larger := False;
                break;
              end;
          end;
          case Larger of
            True: inc(Temp);
          end;
        end;
        inc(Count);
        SetLength(Selector[Key], Count);
        if Temp < Count then
{$IFDEF CLR}
          System.array.Copy(Selector[Key], Temp, Selector[Key], Temp + 1, Count - Temp - 1);
{$ELSE}
          System.Move(Selector[Key][Temp], Selector[Key][Temp + 1], (Count - Temp - 1) * SizeOf(Pointer));
{$ENDIF}
        Selector[Key][Temp] := Item;
      end;
  end;
end;

procedure TmwLexKeyList.Clear;
var
  I, J: Integer;
begin
  for I := 0 to 255 do
    if Selector[I] <> nil then
    begin
      for J := 0 to Length(Selector[I]) - 1 do
      begin
        Selector[I][J].S := '';
        Selector[I][J].Free;
      end;
      SetLength(Selector[I], 0);
    end;
end;

function TmwLexKeyList.Compare(const aStart: Integer; const S: AnsiString): Boolean;
var
  I, J: Integer;
begin
  Result := False;
  J := Length(S);
  if Owner.Run - aStart = J then
  begin
    Result := True;
    dec(J);
    for I := Owner.Run - 2 downto aStart + 1 do
      if Owner.UpperMap[Ord(Owner.Buf[I])] <> Owner.UpperMap[Ord(S[J])] then
      begin
        Result := False;
        Exit;
      end else dec(J);
  end;
end;

constructor TmwLexKeyList.Create(aOwner: TmwD4NLexer);
begin
  inherited Create;
  Owner := aOwner;
end;

destructor TmwLexKeyList.Destroy;
begin
  Clear;
  inherited Destroy;
end;

procedure TmwLexKeyList.Hash(const aStart: Integer);
var
  Key: Byte;
  Count, Value, Value2: Integer;
{$IFDEF CLR}
  Sub: TmwLexSubKeyList;
{$ELSE}
  Sub: PmwLexSubKeyList;
{$ENDIF}
  Item: TmwLexKeyListItem;
  First, Last, Temp: Integer;
begin
  Key := Owner.UpperMap[Ord(Owner.Buf[aStart])];
  Count := Length(Selector[Key]);
{$IFDEF CLR}
  Sub := Selector[Key];
{$ELSE}
  Sub := Pointer(Selector[Key]);
{$ENDIF}
  begin
    First := 0;
    Last := Count - 1;
    Temp := -1;
    Value := Owner.UpperMap[Ord(Owner.Buf[Owner.Run - 1])] shl (Owner.Run - aStart);
    while First <= Last do
    begin
      Temp := (First + Last) div 2;
      Value2 := Sub[Temp].HashValue;
      if Value < Value2 then
        Last := Temp - 1
      else
        if Value > Value2 then
          First := Temp + 1
        else Break;
    end;
    if (Temp > -1) and (Value = Sub[Temp].HashValue) then
    begin
      dec(Temp);
      while (Temp > -1) and (Value = Sub[Temp].HashValue) do
        dec(Temp);
      inc(Temp);
      while (Temp < Count) and (Value = Sub[Temp].HashValue) do
        case Compare(aStart, Sub[Temp].S) of
          True:
            begin
              Item := Sub[Temp];
              Owner.Id := Item.Id;
              Owner.ExId := Item.ExId;
              Exit;
            end;
          False: inc(Temp);
        end;
    end;
  end;
end;

{ TmwD4NLexer }

procedure TmwD4NLexer.AnsiCommentHandler;
begin
  Id := leAnsiComment;
  while Run < TheEnd do
  begin
    case Buf[Run] of
      #10: Exit;

      #13:
        case (Run + 1 < TheEnd) and (Buf[Run + 1] = #10) of
          True: Exit;
          False: inc(Run);
        end;

      '*':
        case (Run + 1 < TheEnd) and (Buf[Run + 1] = ')') of
          True:
            begin
              inc(Run, 2);
              Range := drNormal;
              Exit;
            end;
          False: inc(Run);
        end;
    else
      case Run + UTF8Width[Buf[Run]] < TheEnd of
        True: inc(Run, UTF8Width[Buf[Run]]);
        False: inc(Run);
      end;
    end;
  end;
end;

procedure TmwD4NLexer.AmpersandHandler;
begin
  inc(Run);
  Id := leAmpersand;
end;

procedure TmwD4NLexer.ApostropheHandler;
begin
  inc(Run);
  Id := leUnterminatedStringConstant;
  while Run < TheEnd do
    case Buf[Run] of
      #10, #13: Break;
      #39:
        begin
          inc(Run);
          case Run < TheEnd of
            True:
              case Buf[Run] = #39 of
                True: inc(Run);
                False:
                  begin
                    Id := leStringConstant;
                    Break;
                  end;
              end;
            False:
              begin
                Id := leStringConstant;
                Break;
              end;
          end;
        end;
    else
      case Run + UTF8Width[Buf[Run]] < TheEnd of
        True: inc(Run, UTF8Width[Buf[Run]]);
        False: inc(Run);
      end;

    end;
end;

procedure TmwD4NLexer.AsciiCircumHandler;
begin
  inc(Run);
  Id := leAsciiCircum;
end;

procedure TmwD4NLexer.AsciiTildeHandler;
begin
  inc(Run);
  Id := leAsciiTilde;
end;

procedure TmwD4NLexer.AssemblerReferenceHandler;
begin
  Id := leAssemblerReference;
  while Run < TheEnd do
  begin
    case Buf[Run] of
      #10: Exit;

      #13:
        case (Run + 1 < TheEnd) and (Buf[Run + 1] = #10) of
          True: Exit;
          False: inc(Run);
        end;

      '}':
        begin
          inc(Run);
          Range := drNormal;
          Exit;
        end;
    else
      case Run + UTF8Width[Buf[Run]] < TheEnd of
        True: inc(Run, UTF8Width[Buf[Run]]);
        False: inc(Run);
      end;
    end;
  end;
end;

procedure TmwD4NLexer.AssignData(Source: TmwLexData);
begin
  if (Source <> nil) then
  begin
    ExId := Source.ExId;
    FileName := Source.FileName;
    Id := Source.Id;
    fLinePosition := Source.LinePosition;
    fLineNumber := Source.LineNumber;
    Buf := Source.Buf;
    Range := Source.Range;
    Run := Source.Run;
    Start := Source.Start;
    TheEnd := Source.TheEnd;
    RecordAlignment := Source.RecordAlignment;
    BooleanDirectives := Source.BooleanDirectives;
  end;
end;

procedure TmwD4NLexer.AssignToData(Dest: TmwLexData);
begin
  if (Dest <> nil) then
  begin
    Dest.ExId := ExId;
    Dest.FileName := FileName;
    Dest.Id := Id;
    Dest.LinePosition := LinePosition;
    Dest.LineNumber := LineNumber;
    Dest.Buf := Buf;
    Dest.Range := Range;
    Dest.Run := Run;
    Dest.Start := Start;
    Dest.TheEnd := TheEnd;
    Dest.RecordAlignment := fRecordAlignment;
    Dest.BooleanDirectives := fBooleanDirectives;
  end;
end;

procedure TmwD4NLexer.AsteriskHandler;
begin
  inc(Run);
  Id := leAsterisk;
  if Range in [drAnsiComment, drAnsiDirective] then
    if Run < TheEnd then
      if Buf[Run] = ')' then
      begin
        inc(Run);
        Id := leEndOfAnsiComment;
        Range := drNormal;
      end;
end;

procedure TmwD4NLexer.AtHandler;
begin
  inc(Run);
  Id := leAt;
  if (Run < TheEnd) and (Buf[Run] = '@') then
  begin
    Id := leDoubleAt;
    inc(Run);
  end;
end;

procedure TmwD4NLexer.BackslashHandler;
begin
  inc(Run);
  Id := leBackSlash;
end;

procedure TmwD4NLexer.BarHandler;
begin
  inc(Run);
  Id := leBar;
end;

procedure TmwD4NLexer.BorlandCommentHandler;
begin
  Id := leBorlandComment;
  while Run < TheEnd do
  begin
    case Buf[Run] of
      #10: Exit;

      #13:
        case (Run + 1 < TheEnd) and (Buf[Run + 1] = #10) of
          True: Exit;
          False: inc(Run);
        end;

      '}':
        begin
          inc(Run);
          Range := drNormal;
          Exit;
        end;
    else
      case Run + UTF8Width[Buf[Run]] < TheEnd of
        True: inc(Run, UTF8Width[Buf[Run]]);
        False: inc(Run);
      end;
    end;
  end;
end;

procedure TmwD4NLexer.ColonHandler;
begin
  inc(Run);
  Id := leColon;
  if (Run < TheEnd) and (Buf[Run] = '=') then
  begin
    Id := leAssignment;
    inc(Run);
  end;
end;

procedure TmwD4NLexer.CommaHandler;
begin
  inc(Run);
  Id := leComma;
end;

constructor TmwD4NLexer.Create;
begin
  inherited Create;
  DirectiveStack := TStack.Create;
  fDefaultDefines := TstringList.Create;
  fDefinedList := TmwBinHash.Create;
  fDeclaredList := TmwBinHash.Create;
  InitializeDirectives;
  InitTables;
  CreateKeyLists;
  InitializeKeyTables;
end;

procedure TmwD4NLexer.CreateKeyLists;
begin
  KeyList := TmwLexKeyList.Create(Self);
  DirectiveKeyList := TmwLexKeyList.Create(Self);
end;

procedure TmwD4NLexer.CRHandler;
begin
  inc(Run);
  Id := leSpace;
  if (Run < TheEnd) and (Buf[Run] = #10) then
  begin
    Id := leLineEnd;
    inc(Run);
    fLinePosition := Start;
    inc(fLineNumber);
  end;
end;

procedure TmwD4NLexer.CurlyCloseHandler;
begin
  inc(Run);
  Id := leCurlyClose;
  if Range in [drBorlandComment, drBorlandDirective] then
    Range := drNormal;
end;

procedure TmwD4NLexer.CurlyOpenHandler;
begin
  inc(Run);
  Id := leCurlyOpen;
  if Range = drNormal then
    case Run < TheEnd of
      True:
        case Buf[Run] of
          '$':
            begin
              inc(Run);
              Id := leBeginOfBorlandDirective;
              Range := drBorlandDirective;
            end;
          '%':
            begin
              inc(Run);
              Id := leAssemblerReference;
              Range := drAssemblerReference;
              AssemblerReferenceHandler;
            end;
        else
          begin
            Range := drBorlandComment;
            BorlandCommentHandler;
          end;
        end;
      False:
        begin
          Range := drBorlandComment;
          BorlandCommentHandler;
        end;
    end;
end;

destructor TmwD4NLexer.Destroy;
begin
  DirectiveStack.Free;
  fDefaultDefines.Free;
  fDefinedList.Free;
  fDeclaredList.Free;
  DestroyKeyLists;
  inherited Destroy;
end;

procedure TmwD4NLexer.DestroyKeyLists;
begin
  KeyList.Free;
  DirectiveKeyList.Free;
end;

procedure TmwD4NLexer.DollarHandler;
begin
  inc(Run);
  Id := leDollar;
  if (Run < TheEnd) and
    ((Buf[Run] >= '0') and (Buf[Run] <= '9')) or
    ((Buf[Run] >= 'a') and (Buf[Run] <= 'f')) or
    ((Buf[Run] >= 'A') and (Buf[Run] <= 'F')) then
  begin
    inc(Run);
    Id := leHexNumber;
    while (Run < TheEnd) and
      ((Buf[Run] >= '0') and (Buf[Run] <= '9')) or
      ((Buf[Run] >= 'a') and (Buf[Run] <= 'f')) or
      ((Buf[Run] >= 'A') and (Buf[Run] <= 'F')) do
      inc(Run);
  end;
end;

procedure TmwD4NLexer.DoubleQuoteHandler;
begin
  inc(Run);
  Id := leDoubleQuote;
end;

procedure TmwD4NLexer.EqualHandler;
begin
  inc(Run);
  Id := leEqual;
end;

function TmwD4NLexer.Equals(const S: AnsiString): Boolean;
var
  I: Integer;
  J: Integer;
begin
  Result := False;
  J := Length(S);
  if Run - Start = J then
    case ExId of
      leUTF8Identifier:
        begin
          Result := True;
          for I := Run - 1 downto Start do
          begin
            if Buf[I] <> S[J] then
            begin
              Result := False;
              break;
            end;
            dec(J);
          end;
          if Result = False then
          begin
{$IFDEF CLR}
            Result :=
              UpperCase(System.Text.Encoding.UTF8.GetString(TBytes(S)))
              =
              UpperCase(System.Text.Encoding.UTF8.GetString(TBytes(Buf), Start, Run - Start));
{$ELSE}
            Result :=
              UpperCase(UTF8Decode(S))
              =
              UpperCase(UTF8Decode(Copy(Buf, Start, Run - Start)));
{$ENDIF}
          end;
        end;
      leIdentifier:
        begin
          Result := True;
          for I := Run - 1 downto Start do
          begin
            if Buf[I] <> S[J] then
              if UpperMap[Ord(Buf[I])] <> UpperMap[Ord(S[J])] then
              begin
                Result := False;
                Exit;
              end;
            dec(J);
          end;
        end;
    else
      begin
        Result := True;
        for I := Run - 1 downto Start do
        begin
          if Buf[I] <> S[J] then
          begin
            Result := False;
            Exit;
          end;
          dec(J);
        end;
      end;
    end;
end;

procedure TmwD4NLexer.ExclamHandler;
begin
  inc(Run);
  Id := leExclam;
end;

function TmwD4NLexer.GetColumn: Integer;
begin
  Result := Start - LinePosition;
end;

function TmwD4NLexer.GetLinePosition: Integer;
begin
  Result := LinePosition;
end;

function TmwD4NLexer.GetRun: Integer;
begin
  Result := Run;
end;

function TmwD4NLexer.GetStart: Integer;
begin
  Result := Start;
end;

function TmwD4NLexer.GetToken: AnsiString;
begin
  SetLength(Result, Run - Start);
{$IFDEF CLR}
  System.array.Copy(Buf, Start, Result, 0, Run - Start);
{$ELSE}
  System.Move(Buf[Start], Result[1], Run - Start);
{$ENDIF}
end;

{$IFDEF CLR}

function TmwD4NLexer.GetTokenWide: string;
{$ELSE}

function TmwD4NLexer.GetTokenWide: WideString;
{$ENDIF}
begin
{$IFDEF CLR}
  Result := System.Text.Encoding.UTF8.GetString(TBytes(Buf), Start, Run - Start);
{$ELSE}
  Result := UTF8Decode(Copy(Buf, Start, Run - Start));
{$ENDIF}
end;

procedure TmwD4NLexer.GreaterHandler;
begin
  inc(Run);
  Id := leGreater;
  if (Run < TheEnd) and (Buf[Run] = '=') then
  begin
    Id := leGreaterOrEqual;
    inc(Run);
  end;
end;

procedure TmwD4NLexer.IdentifierHandler;
begin
  inc(Run);
  Id := leIdentifier;
  ExId := leIdentifier;
  while (Run < TheEnd) do
    case InIdentifiers[Buf[Run]] of
      True: inc(Run);
      False: Break;
    end;
  case InInternationalIdentifiers[Buf[Run]] of
    True: InternationalIdentifierHandler;
    False:
      case (Run + UTF8Width[Buf[Run]] < TheEnd) and (Buf[Run] > #191) of
        True: UTF8IdentifierHandler;
      else
        case Range of
          drNormal: KeyList.Hash(Start);
          drAnsiDirective: DirectiveKeyList.Hash(Start);
          drBorlandDirective: DirectiveKeyList.Hash(Start);
        end;
      end;
  end;
end;

procedure TmwD4NLexer.InternationalIdentifierHandler;
begin
  inc(Run);
  Id := leIdentifier;
  ExId := leInternationalIdentifier;
  while (Run < TheEnd) do
    case InInternationalIdentifiers[Buf[Run]] of
      True: inc(Run);
      False: Break;
    end;
  if (Run + UTF8Width[Buf[Run]] < TheEnd) and (Buf[Run] > #191) then
    UTF8IdentifierHandler;
end;

procedure TmwD4NLexer.InitializeDirectives;
begin
  fApptype := atGUI;
  fRecordAlignment := 8;
  fImageBase := $00400000;
  fMinStackSize := 16384;
  fMaxStackSize := 1048576;
  fResourceReserve := IntToStr(1048576);
  fMinEnumSize := 1;
  Include(fBooleanDirectives, bdAlign);
  Include(fBooleanDirectives, bdAssertions);
  Exclude(fBooleanDirectives, bdBoolEval);
  Include(fBooleanDirectives, bdDebugInfo);
  Include(fBooleanDirectives, bdDefinitionInfo);
  Exclude(fBooleanDirectives, bdDenyPackageUnit);
  Exclude(fBooleanDirectives, bdDesignOnly);
  Include(fBooleanDirectives, bdExtendedSyntax);
  Include(fBooleanDirectives, bdImportedData);
  Include(fBooleanDirectives, bdLongStrings);
  Include(fBooleanDirectives, bdHints);
  Include(fBooleanDirectives, bdIOChecks);
  Include(fBooleanDirectives, bdImplicitBuild);
  Exclude(fBooleanDirectives, bdWriteableConst);
  Include(fBooleanDirectives, bdLocalSymbols);
  Exclude(fBooleanDirectives, bdTypeInfo);
  Exclude(fBooleanDirectives, bdObjExportAll);
  Include(fBooleanDirectives, bdOptimization);
  Include(fBooleanDirectives, bdOpenStrings);
  Exclude(fBooleanDirectives, bdOverFlowChecks);
  Exclude(fBooleanDirectives, bdRangeChecks);
  Exclude(fBooleanDirectives, bdStackFrames);
  Exclude(fBooleanDirectives, bdWeakPackageUnit);
  Include(fBooleanDirectives, bdWarnings);
  Include(fBooleanDirectives, bdSymbol_Platform);
  Include(fBooleanDirectives, bdSymbol_Library);
  Include(fBooleanDirectives, bdSymbol_Deprecated);
  Include(fBooleanDirectives, bdUnit_Deprecated);
  Include(fBooleanDirectives, bdUnit_Library);
  Include(fBooleanDirectives, bdUnit_Platform);
  Include(fBooleanDirectives, bdVarStringChecks);
  Exclude(fBooleanDirectives, bdSafeDivide);
  Exclude(fBooleanDirectives, bdTypedAddress);
  Exclude(fBooleanDirectives, bdStackChecks);
  Exclude(fBooleanDirectives, bdRunOnly);
  Exclude(fBooleanDirectives, bdRealCompatibility);
end;

procedure TmwD4NLexer.InitializeKeyTables;
begin
  {KeyWords}
  KeyList.Add('And', leAnd, leDelphiKeyWord);
  KeyList.Add('Array', leArray, leDelphiKeyWord);
  KeyList.Add('As', leAs, leDelphiKeyWord);
  KeyList.Add('Asm', leAsm, leDelphiKeyWord);
  KeyList.Add('Begin', leBegin, leDelphiKeyWord);
  KeyList.Add('Case', leCase, leDelphiKeyWord);
  KeyList.Add('Class', leClass, leDelphiKeyWord);
  KeyList.Add('Const', leConst, leDelphiKeyWord);
  KeyList.Add('Constructor', leConstructor, leDelphiKeyWord);
  KeyList.Add('Destructor', leDestructor, leDelphiKeyWord);
  KeyList.Add('Dispinterface', leDispinterface, leDelphiKeyWord);
  KeyList.Add('Div', leDiv, leDelphiKeyWord);
  KeyList.Add('Do', leDo, leDelphiKeyWord);
  KeyList.Add('Downto', leDownto, leDelphiKeyWord);
  KeyList.Add('Else', leElse, leDelphiKeyWord);
  KeyList.Add('End', leEnd, leDelphiKeyWord);
  KeyList.Add('Except', leExcept, leDelphiKeyWord);
  KeyList.Add('Exports', leExports, leDelphiKeyWord);
  KeyList.Add('File', leFile, leDelphiKeyWord);
  KeyList.Add('Final', leFinal, leDelphiKeyWord);
  KeyList.Add('Finalization', leFinalization, leDelphiKeyWord);
  KeyList.Add('Finally', leFinally, leDelphiKeyWord);
  KeyList.Add('For', leFor, leDelphiKeyWord);
  KeyList.Add('Function', leFunction, leDelphiKeyWord);
  KeyList.Add('Goto', leGoto, leDelphiKeyWord);
  KeyList.Add('If', leIf, leDelphiKeyWord);
  KeyList.Add('Implementation', leImplementation, leDelphiKeyWord);
  KeyList.Add('In', leIn, leDelphiKeyWord);
  KeyList.Add('Inherited', leInherited, leDelphiKeyWord);
  KeyList.Add('Initialization', leInitialization, leDelphiKeyWord);
  KeyList.Add('Inline', leInline, leDelphiKeyWord);
  KeyList.Add('Interface', leInterface, leDelphiKeyWord);
  KeyList.Add('Is', leIs, leDelphiKeyWord);
  KeyList.Add('Label', leLabel, leDelphiKeyWord);
  KeyList.Add('Library', leLibrary, leDelphiDirective);
  KeyList.Add('Mod', leMod, leDelphiKeyWord);
  KeyList.Add('Nil', leNil, leDelphiKeyWord);
  KeyList.Add('Not', leNot, leDelphiKeyWord);
  KeyList.Add('Object', leObject, leDelphiKeyWord);
  KeyList.Add('Of', leOf, leDelphiKeyWord);
  KeyList.Add('Or', leOr, leDelphiKeyWord);
  KeyList.Add('Packed', lePacked, leDelphiKeyWord);
  KeyList.Add('Procedure', leProcedure, leDelphiKeyWord);
  KeyList.Add('Program', leProgram, leDelphiKeyWord);
  KeyList.Add('Property', leProperty, leDelphiKeyWord);
  KeyList.Add('Raise', leRaise, leDelphiKeyWord);
  KeyList.Add('Record', leRecord, leDelphiKeyWord);
  KeyList.Add('Repeat', leRepeat, leDelphiKeyWord);
  KeyList.Add('Resourcestring', leResourcestring, leDelphiKeyWord);
  KeyList.Add('Sealed', leSealed, leDelphiKeyWord);
  KeyList.Add('Set', leSet, leDelphiKeyWord);
  KeyList.Add('Shl', leShl, leDelphiKeyWord);
  KeyList.Add('Shr', leShr, leDelphiKeyWord);
  KeyList.Add('String', leString, leDelphiKeyWord);
  KeyList.Add('Then', leThen, leDelphiKeyWord);
  KeyList.Add('Threadvar', leThreadvar, leDelphiKeyWord);
  KeyList.Add('To', leTo, leDelphiKeyWord);
  KeyList.Add('Try', leTry, leDelphiKeyWord);
  KeyList.Add('Type', leType, leDelphiKeyWord);
  KeyList.Add('Unit', leUnit, leDelphiKeyWord);
  KeyList.Add('Until', leUntil, leDelphiKeyWord);
  KeyList.Add('Uses', leUses, leDelphiKeyWord);
  KeyList.Add('Var', leVar, leDelphiKeyWord);
  KeyList.Add('While', leWhile, leDelphiKeyWord);
  KeyList.Add('With', leWith, leDelphiKeyWord);
  KeyList.Add('Xor', leXor, leDelphiKeyWord);

  {Directives}
  KeyList.Add('Absolute', leIdentifier, leAbsolute);
  KeyList.Add('Abstract', leIdentifier, leAbstract);
  KeyList.Add('Assembler', leIdentifier, leAssembler);
  KeyList.Add('At', leIdentifier, leAt);
  KeyList.Add('Automated', leIdentifier, leAutomated);
  KeyList.Add('Cdecl', leIdentifier, leCdecl);
  KeyList.Add('Contains', leIdentifier, leContains);
  KeyList.Add('Default', leIdentifier, leDefault);
  KeyList.Add('Deprecated', leIdentifier, leDeprecated);
  KeyList.Add('Dispid', leIdentifier, leDispid);
  KeyList.Add('Dynamic', leIdentifier, leDynamic);
  KeyList.Add('Experimental', leIdentifier, leExperimental);
  KeyList.Add('Export', leIdentifier, leExport);
  KeyList.Add('External', leIdentifier, leExternal);
  KeyList.Add('Far', leIdentifier, leFar);
  KeyList.Add('Forward', leIdentifier, leForward);
  KeyList.Add('Helper', leIdentifier, leHelper);
  KeyList.Add('Implements', leIdentifier, leImplements);
  KeyList.Add('Index', leIdentifier, leIndex);
  KeyList.Add('Local', leIdentifier, leLocal);
  KeyList.Add('Message', leIdentifier, leMessage);
  KeyList.Add('Name', leIdentifier, leName);
  KeyList.Add('Near', leIdentifier, leNear);
  KeyList.Add('Nodefault', leIdentifier, leNodefault);
  KeyList.Add('On', leIdentifier, leOn);
  KeyList.Add('Out', leIdentifier, leOut);
  KeyList.Add('Overload', leIdentifier, leOverload);
  KeyList.Add('Override', leIdentifier, leOverride);
  KeyList.Add('Package', leIdentifier, lePackage);
  KeyList.Add('Pascal', leIdentifier, lePascal);
  KeyList.Add('Platform', leIdentifier, lePlatform);
  KeyList.Add('Private', leIdentifier, lePrivate);
  KeyList.Add('Protected', leIdentifier, leProtected);
  KeyList.Add('Public', leIdentifier, lePublic);
  KeyList.Add('Published', leIdentifier, lePublished);
  KeyList.Add('Read', leIdentifier, leRead);
  KeyList.Add('Readonly', leIdentifier, leReadonly);
  KeyList.Add('Register', leIdentifier, leRegister);
  KeyList.Add('Reintroduce', leIdentifier, leReintroduce);
  KeyList.Add('Requires', leIdentifier, leRequires);
  KeyList.Add('Resident', leIdentifier, leResident);
  KeyList.Add('Safecall', leIdentifier, leSafecall);
  KeyList.Add('Stdcall', leIdentifier, leStdcall);
  KeyList.Add('Stored', leIdentifier, leStored);
  KeyList.Add('Strict', leIdentifier, leStrict);
  KeyList.Add('Varargs', leIdentifier, leVarargs);
  KeyList.Add('Virtual', leIdentifier, leVirtual);
  KeyList.Add('Write', leIdentifier, leWrite);
  KeyList.Add('Writeonly', leIdentifier, leWriteonly);

  {Additional}
  DirectiveKeyList.Add('A', leA, leIdentifier);
  DirectiveKeyList.Add('B', leB, leIdentifier);
  DirectiveKeyList.Add('C', leC, leIdentifier);
  DirectiveKeyList.Add('D', leD, leIdentifier);
  DirectiveKeyList.Add('E', leE, leIdentifier);
  DirectiveKeyList.Add('G', leG, leIdentifier);
  DirectiveKeyList.Add('H', leH, leIdentifier);
  DirectiveKeyList.Add('I', leI, leIdentifier);
  DirectiveKeyList.Add('J', leJ, leIdentifier);
  DirectiveKeyList.Add('L', leL, leIdentifier);
  DirectiveKeyList.Add('M', leM, leIdentifier);
  DirectiveKeyList.Add('O', leO, leIdentifier);
  DirectiveKeyList.Add('P', leP, leIdentifier);
  DirectiveKeyList.Add('Q', leQ, leIdentifier);
  DirectiveKeyList.Add('R', leR, leIdentifier);
  DirectiveKeyList.Add('S', leS, leIdentifier);
  DirectiveKeyList.Add('T', leT, leIdentifier);
  DirectiveKeyList.Add('U', leU, leIdentifier);
  DirectiveKeyList.Add('V', leV, leIdentifier);
  DirectiveKeyList.Add('W', leW, leIdentifier);
  DirectiveKeyList.Add('X', leX, leIdentifier);
  DirectiveKeyList.Add('Y', leY, leIdentifier);
  DirectiveKeyList.Add('Z', leZ, leIdentifier);
  DirectiveKeyList.Add('And', leAndDirective, leIdentifier);
  DirectiveKeyList.Add('Off', leOff, leIdentifier);
  DirectiveKeyList.Add('YD', leYD, leIdentifier);
  DirectiveKeyList.Add('On', leOnDirective, leIdentifier);
  DirectiveKeyList.Add('If', leIfDirective, leIdentifier);
  DirectiveKeyList.Add('IfDef', leIfDef, leIdentifier);
  DirectiveKeyList.Add('Or', leOrDirective, leIdentifier);
  DirectiveKeyList.Add('GUI', leGUI, leIdentifier);
  DirectiveKeyList.Add('EndIf', leEndIf, leIdentifier);
  DirectiveKeyList.Add('IfEnd', leIfEnd, leIdentifier);
  DirectiveKeyList.Add('Else', leElseDirective, leIdentifier);
  DirectiveKeyList.Add('Define', leDefine, leIdentifier);
  DirectiveKeyList.Add('Align', leAlign, leIdentifier);
  DirectiveKeyList.Add('IfNDef', leIfNDef, leIdentifier);
  DirectiveKeyList.Add('Link', leLink, leIdentifier);
  DirectiveKeyList.Add('Defined', leDefined, leIdentifier);
  DirectiveKeyList.Add('UnDef', leUnDef, leIdentifier);
  DirectiveKeyList.Add('Declared', leDeclared, leIdentifier);
  DirectiveKeyList.Add('ElseIf', leElseIf, leIdentifier);
  DirectiveKeyList.Add('ImageBase', leImageBase, leIdentifier);
  DirectiveKeyList.Add('IfOpt', leIfOpt, leIdentifier);
  DirectiveKeyList.Add('SoName', leSoName, leIdentifier);
  DirectiveKeyList.Add('Include', leInclude, leIdentifier);
  DirectiveKeyList.Add('Hints', leHints, leIdentifier);
  DirectiveKeyList.Add('NoDefine', leNoDefine, leIdentifier);
  DirectiveKeyList.Add('IOChecks', leIOChecks, leIdentifier);
  DirectiveKeyList.Add('DebugInfo', leDebugInfo, leIdentifier);
  DirectiveKeyList.Add('Console', leConsole, leIdentifier);
  DirectiveKeyList.Add('Booleval', leBooleval, leIdentifier);
  DirectiveKeyList.Add('HPPEmit', leHPPEmit, leIdentifier);
  DirectiveKeyList.Add('RangeChecks', leRangeChecks, leIdentifier);
  DirectiveKeyList.Add('NoInclude', leNoInclude, leIdentifier);
  DirectiveKeyList.Add('Apptype', leApptype, leIdentifier);
  DirectiveKeyList.Add('SafeDivide', leSafeDivide, leIdentifier);
  DirectiveKeyList.Add('Resource', leResource, leIdentifier);
  DirectiveKeyList.Add('Warn', leWarn, leIdentifier);
  DirectiveKeyList.Add('Warnings', leWarnings, leIdentifier);
  DirectiveKeyList.Add('TypeInfo', leTypeInfo, leIdentifier);
  DirectiveKeyList.Add('SoPrefix', leSoPrefix, leIdentifier);
  DirectiveKeyList.Add('StackFrames', leStackFrames, leIdentifier);
  DirectiveKeyList.Add('RunOnly', leRunOnly, leIdentifier);
  DirectiveKeyList.Add('SoSuffix', leSoSuffix, leIdentifier);
  DirectiveKeyList.Add('ReferenceInfo', leReferenceInfo, leIdentifier);
  DirectiveKeyList.Add('DesignOnly', leDesignOnly, leIdentifier);
  DirectiveKeyList.Add('Extension', leExtension, leIdentifier);
  DirectiveKeyList.Add('ImportedData', leImportedData, leIdentifier);
  DirectiveKeyList.Add('Description', leDescription, leIdentifier);
  DirectiveKeyList.Add('SoVersion', leSoVersion, leIdentifier);
  DirectiveKeyList.Add('Assertions', leAssertions, leIdentifier);
  DirectiveKeyList.Add('ImplicitBuild', leImplicitBuild, leIdentifier);
  DirectiveKeyList.Add('TypedAddress', leTypedAddress, leIdentifier);
  DirectiveKeyList.Add('LocalSymbols', leLocalSymbols, leIdentifier);
  DirectiveKeyList.Add('MinEnumSize', leMinEnumSize, leIdentifier);
  DirectiveKeyList.Add('WeakPackageUnit', leWeakPackageUnit, leIdentifier);
  DirectiveKeyList.Add('DefinitionInfo', leDefinitionInfo, leIdentifier);
  DirectiveKeyList.Add('MinStackSize', leMinStackSize, leIdentifier);
  DirectiveKeyList.Add('ObjExportAll', leObjExportAll, leIdentifier);
  DirectiveKeyList.Add('MaxStackSize', leMaxStackSize, leIdentifier);
  DirectiveKeyList.Add('LongStrings', leLongStrings, leIdentifier);
  DirectiveKeyList.Add('DenyPackageUnit', leDenyPackageUnit, leIdentifier);
  DirectiveKeyList.Add('ExternalSym', leExternalSym, leIdentifier);
  DirectiveKeyList.Add('OpenStrings', leOpenStrings, leIdentifier);
  DirectiveKeyList.Add('OverFlowChecks', leOverFlowChecks, leIdentifier);
  DirectiveKeyList.Add('WriteableConst', leWriteableConst, leIdentifier);
  DirectiveKeyList.Add('Optimization', leOptimization, leIdentifier);
  DirectiveKeyList.Add('VarStringChecks', leVarStringChecks, leIdentifier);
  DirectiveKeyList.Add('ExtendedSyntax', leExtendedSyntax, leIdentifier);
  DirectiveKeyList.Add('RealCompatibility', leRealCompatibility, leIdentifier);
  DirectiveKeyList.Add('ResourceReserve', leResourceReserve, leIdentifier);
  DirectiveKeyList.Add('Region', leRegion, leIdentifier);
  DirectiveKeyList.Add('EndRegion', leEndRegion, leIdentifier);
  DirectiveKeyList.Add('StackChecks', leStackChecks, leIdentifier);
  DirectiveKeyList.Add('Symbol_Platform', leSymbol_Platform, leIdentifier);
  DirectiveKeyList.Add('Symbol_Library', leSymbol_Library, leIdentifier);
  DirectiveKeyList.Add('Symbol_Deprecated', leSymbol_Deprecated, leIdentifier);
  DirectiveKeyList.Add('Unit_Deprecated', leUnit_Deprecated, leIdentifier);
  DirectiveKeyList.Add('Unit_Library', leUnit_Library, leIdentifier);
  DirectiveKeyList.Add('Unit_Platform', leUnit_Platform, leIdentifier);
end;

procedure TmwD4NLexer.InitTables;
var
  I: AnsiChar;
  S: AnsiString;
begin
  for I := #0 to #255 do
  begin
    case I in ['0'..'9', 'A'..'Z', 'a'..'z', '_'] of
      True:
        begin
          InIdentifiers[I] := True;
          InInternationalIdentifiers[I] := True;
        end;
      False:
        begin
          InIdentifiers[I] := False;
          case I < #192 of
            True:
              case IsCharAlphaA(I) of
                True: InInternationalIdentifiers[I] := True;
                False: InInternationalIdentifiers[I] := False;
              end;
            False: InInternationalIdentifiers[I] := False;
          end
        end;
    end;
    S := AnsiUpperCase(I);
    case S <> '' of
      True: UpperMap[Ord(I)] := Ord(S[1]);
      False: UpperMap[Ord(I)] := Ord(I);
    end;
  end;
end;

procedure TmwD4NLexer.LessHandler;
begin
  inc(Run);
  Id := leLess;
  if Run < TheEnd then
    case Buf[Run] of
      '=':
        begin
          Id := leLessOrEqual;
          inc(Run);
        end;
      '>':
        begin
          Id := leNotEqual;
          inc(Run);
        end;
    end;
end;

procedure TmwD4NLexer.LFHandler;
begin
  inc(Run);
  Id := leLF;
  fLinePosition := Start;
  inc(fLineNumber);
end;

procedure TmwD4NLexer.MinusHandler;
begin
  inc(Run);
  Id := leMinus;
end;

procedure TmwD4NLexer.Next;
begin
  Start := Run;
  ExId := leUnknown;
  case Run < TheEnd of
    True:
      case Range of
        drNormal,
          drAnsiDirective,
          drBorlandDirective:
          case Buf[Run] of
            #0..#9: SpaceHandler;
            #10: LFHandler;
            #11, #12: SpaceHandler;
            #13: CRHandler;
            #14..#32: SpaceHandler;
            '!': ExclamHandler;
            '"': DoubleQuoteHandler;
            '#': SharpHandler;
            '$': DollarHandler;
            '%': PercentHandler;
            '&': AmpersandHandler;
            '''': ApostropheHandler;
            '(': RoundOpenHandler;
            ')': RoundCloseHandler;
            '*': AsteriskHandler;
            '+': PlusHandler;
            ',': CommaHandler;
            '-': MinusHandler;
            '.': PeriodHandler;
            '/': SlashHandler;
            '0'..'9': NumberHandler;
            ':': ColonHandler;
            ';': SemicolonHandler;
            '<': LessHandler;
            '=': EqualHandler;
            '>': GreaterHandler;
            '?': QuestionHandler;
            '@': AtHandler;
            'A'..'Z': IdentifierHandler;
            '[': SquareOpenHandler;
            '\': BackslashHandler;
            ']': SquareCloseHandler;
            '^': AsciiCircumHandler;
            '_': IdentifierHandler;
            '`': QuoteLeftHandler;
            'a'..'z': IdentifierHandler;
            '{': CurlyOpenHandler;
            '|': BarHandler;
            '}': CurlyCloseHandler;
            '~': AsciiTildeHandler;
          else
            case InInternationalIdentifiers[Buf[Run]] of
              True: InternationalIdentifierHandler;
              False:
                case (Buf[Run] > #191) and (Run + UTF8Width[Buf[Run]] <= TheEnd) of
                  True: InternationalIdentifierHandler;
                  False: UnknownHandler;
                end;
            end;
          end;
        drAssemblerReference: NextAssemblerReference;
        drAnsiComment: NextAnsiComment;
        drBorlandComment: NextBorlandComment;
      end;
    False: Id := leAtEnd;
  end
end;

procedure TmwD4NLexer.NextNoSpace;
begin
  repeat
    Next;
  until (Id <> leSpace) and (Id <> leLineEnd) and (Id <> leLF);
end;

procedure TmwD4NLexer.NextToken;
begin
  Next;
end;

procedure TmwD4NLexer.NextAnsiComment;
begin
  Start := Run;
  ExId := leUnknown;
  case Run < TheEnd of
    True:
      case Buf[Run] of
        #10: LFHandler;
        #13: CRHandler;
      else
        AnsiCommentHandler;
      end;
    False: Id := leAtEnd;
  end
end;

procedure TmwD4NLexer.NextAssemblerReference;
begin
  Start := Run;
  ExId := leUnknown;
  case Run < TheEnd of
    True:
      case Buf[Run] of
        #10: LFHandler;
        #13: CRHandler;
      else
        AssemblerReferenceHandler;
      end;
    False: Id := leAtEnd;
  end
end;

procedure TmwD4NLexer.NextBorlandComment;
begin
  Start := Run;
  ExId := leUnknown;
  case Run < TheEnd of
    True:
      case Buf[Run] of
        #10: LFHandler;
        #13: CRHandler;
      else
        BorlandCommentHandler;
      end;
    False: Id := leAtEnd;
  end
end;

procedure TmwD4NLexer.NumberHandler;
begin
  Id := leNumber;
  inc(Run);
  while (Run < TheEnd) and ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
    case Run < TheEnd of
      True: inc(Run);
      False: Break;
    end;
  if Run < TheEnd then
    case Buf[Run] of
      '.':
        begin
          inc(Run);
          Id := leFloat;
          if Run < TheEnd then
            case Buf[Run] of
              'E', 'e':
                if Run + 1 < TheEnd then
                  case Buf[Run + 1] of
                    '+', '-':
                      if Run + 2 < TheEnd then
                        if ((Buf[Run + 2] >= '0') and (Buf[Run + 2] <= '9')) then
                        begin
                          inc(Run, 2);
                          while (Run < TheEnd) and
                            ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                            inc(Run);
                        end;
                    '0'..'9':
                      if Run + 1 < TheEnd then
                      begin
                        inc(Run, 2);
                        while (Run < TheEnd) and
                          ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                          inc(Run);
                      end;
                  end;
              '0'..'9':
                begin
                  while (Run < TheEnd) and
                    ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                    inc(Run);
                  if Run < TheEnd then
                    case Buf[Run] of
                      'E', 'e':
                        if Run + 1 < TheEnd then
                          case Buf[Run + 1] of
                            '+', '-':
                              if Run + 1 < TheEnd then
                                if ((Buf[Run + 2] >= '0') and (Buf[Run + 2] <= '9')) then
                                begin
                                  inc(Run, 2);
                                  while (Run < TheEnd) and
                                    ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                                    inc(Run);
                                end;
                            '0'..'9':
                              if Run + 1 < TheEnd then
                              begin
                                inc(Run, 2);
                                while (Run < TheEnd) and
                                  ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                                  inc(Run);
                              end;
                          end;
                    end;
                end;
            end;
        end;
      'E', 'e':
        if Run < TheEnd then
          case Buf[Run + 1] of
            '+', '-':
              if Run + 2 < TheEnd then
                if ((Buf[Run + 2] >= '0') and (Buf[Run + 2] <= '9')) then
                begin
                  inc(Run, 3);
                  Id := leFloat;
                  while (Run < TheEnd) and
                    ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                    inc(Run);
                end;
            '0'..'9':
              if Run + 1 < TheEnd then
              begin
                inc(Run, 2);
                Id := leFloat;
                while (Run < TheEnd) and
                  ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
                  inc(Run);
              end;
          end;
    end;
end;

procedure TmwD4NLexer.PercentHandler;
begin
  inc(Run);
  Id := lePercent;
end;

procedure TmwD4NLexer.PeriodHandler;
begin
  inc(Run);
  Id := lePeriod;
  if Run < TheEnd then
    case Buf[Run] of
      '.':
        begin
          inc(Run);
          Id := leDotDot;
        end;
      ')':
        begin
          inc(Run);
          Id := leSquareClose;
        end;
    end;
end;

procedure TmwD4NLexer.PlusHandler;
begin
  inc(Run);
  Id := lePlus;
end;

procedure TmwD4NLexer.QuestionHandler;
begin
  inc(Run);
  Id := leQuestion;
end;

procedure TmwD4NLexer.QuoteLeftHandler;
begin
  inc(Run);
  Id := leQuoteLeft;
end;

function TmwD4NLexer.RetrieveIncludeFileName: AnsiString;
begin
  if Buf[Run] = '.' then
    inc(Run);
  while Buf[Run] in ['A'..'Z', 'a'..'z'] do
    inc(Run);
  Result := Token;
end;

function TmwD4NLexer.RetrieveResourceFileName: AnsiString;
begin
  Result := '';
  case Buf[Run] of
    '''':
      begin
        SetLength(Result, Run - Start - 2);
{$IFDEF CLR}
        System.array.Copy(Buf, Start + 1, Result, 0, Run - Start - 2);
{$ELSE}
        System.Move(Buf[Start + 1], Result[1], Run - Start - 2);
{$ENDIF}
      end;
    '*':
      begin
        inc(Run);
        Start := Run;
        if Buf[Run] = '.' then
          inc(Run);
        while Buf[Run] in ['A'..'Z', 'a'..'z'] do
          inc(Run);
        Result := FileName + Token;
      end;
  else
    if Id = leIdentifier then
    begin
      if Buf[Run] = '.' then
        inc(Run);
      while Buf[Run] in ['A'..'Z', 'a'..'z'] do
        inc(Run);
      Result := Token;
    end;
  end;
end;

procedure TmwD4NLexer.RoundCloseHandler;
begin
  inc(Run);
  Id := leRoundClose;
end;

procedure TmwD4NLexer.RoundOpenHandler;
begin
  inc(Run);
  Id := leRoundOpen;
  if Range = drNormal then
    if Run < TheEnd then
      case Buf[Run] of
        '.':
          begin
            inc(Run);
            Id := leSquareOpen;
          end;
        '*':
          begin
            inc(Run);
            Id := leAnsiComment;
            if Run < TheEnd then
              case Buf[Run] = '$' of
                True:
                  begin
                    inc(Run);
                    Range := drAnsiDirective;
                    Id := leBeginOfAnsiDirective;
                  end;
                False:
                  begin
                    Range := drAnsiComment;
                    AnsiCommentHandler;
                  end;
              end;
          end;
      end;
end;

procedure TmwD4NLexer.SemicolonHandler;
begin
  inc(Run);
  Id := leSemiColon;
end;

{$IFDEF CLR}

procedure TmwD4NLexer.SetBuf(const aBuf: array of AnsiChar);
{$ELSE}

procedure TmwD4NLexer.SetBuf(const aBuf: PChar; Len: Integer);
{$ENDIF}
begin
  Buf := aBuf;
  Run := 0;
{$IFDEF CLR}
  TheEnd := Length(aBuf);
{$ELSE}
  TheEnd := Len;
{$ENDIF}
  Id := leUnknown;
  fLinePosition := 0;
  fLineNumber := 0;
  NextToken;
end;

procedure TmwD4NLexer.SetDefaultDefines(Value: TStringList);
var
  I: Integer;
begin
  if Value = nil then Exit;
  fDefaultDefines.Assign(Value);
  for I := 0 to fDefaultDefines.Count - 1 do
    fDefinedList.Add(fDefaultDefines[I]);
end;

procedure TmwD4NLexer.SetDescription(Value: AnsiString);
begin
  fDescription := Value;
end;

procedure TmwD4NLexer.SetExtension(Value: AnsiString);
begin
  fExtension := Value;
end;

procedure TmwD4NLexer.SetExternalSym(Value: AnsiString);
begin
  fExternalSym := Value;
end;

procedure TmwD4NLexer.SetHppEmit(Value: AnsiString);
begin
  fHppEmit := Value;
end;

procedure TmwD4NLexer.SetImageBase(Value: Integer);
begin
  fImageBase := Value;
end;

procedure TmwD4NLexer.SetResourceReserve(Value: AnsiString);
begin
  fResourceReserve := Value;
end;

procedure TmwD4NLexer.SetMaxStackSize(Value: Integer);
begin
  fMaxStackSize := Value;
end;

procedure TmwD4NLexer.SetMinStackSize(Value: Integer);
begin
  fMinStackSize := Value;
end;

procedure TmwD4NLexer.SetMinEnumSize(Value: Integer);
begin
  if (Value = 1) or (Value = 2) or (Value = 4) then
    fMinEnumSize := Value;
end;

procedure TmwD4NLexer.SetNoDefine(Value: AnsiString);
begin
  fNoDefine := Value;
end;

procedure TmwD4NLexer.SetNoInclude(Value: AnsiString);
begin
  fNoInclude := Value;
end;

procedure TmwD4NLexer.SetResFileName(Value: AnsiString);
begin
  fResFileName := Value;
end;

procedure TmwD4NLexer.SetRcFileName(Value: AnsiString);
begin
  fRcFileName := Value;
end;

procedure TmwD4NLexer.SetRegionString(Value: AnsiString);
begin
  fRegionString := Value;
end;

procedure TmwD4NLexer.SharpHandler;
begin
  inc(Run);
  Id := leSharp;
  if Run < TheEnd then
    case Buf[Run] of
      '$':
        begin
          inc(Run);
          Id := leCharHexConstant;
          while (Run < TheEnd) and
            ((Buf[Run] >= '0') and (Buf[Run] <= '9')) or
            ((Buf[Run] >= 'a') and (Buf[Run] <= 'f')) or
            ((Buf[Run] >= 'A') and (Buf[Run] <= 'F')) do
            inc(Run);
        end;
      '0'..'9':
        begin
          inc(Run);
          Id := leCharConstant;
          while (Run < TheEnd) and
            ((Buf[Run] >= '0') and (Buf[Run] <= '9')) do
            inc(Run);
        end;
    end;
end;

procedure TmwD4NLexer.SlashHandler;
begin
  inc(Run);
  Id := leSlash;
  if Range = drNormal then
    case Run < TheEnd of
      True:
        if Buf[Run] = '/' then
        begin
          inc(Run);
          Id := leSlashComment;
          while Run < TheEnd do
            case Buf[Run] of
              #10: Break;
              #13: case (Run < TheEnd) and (Buf[Run + 1] = #10) of
                  True: Break;
                  False: inc(Run);
                end;
            else
              inc(Run);
            end;
        end;
    end;
end;

procedure TmwD4NLexer.SpaceHandler;
begin
  inc(Run);
  Id := leSpace;
  while (Run < TheEnd) and
    (((Buf[Run] >= #0) and (Buf[Run] < #10)) or
    (Buf[Run] = #11) or
    (Buf[Run] = #12) or
    ((Buf[Run] = #13) and (Run + 1 < TheEnd) and (Buf[Run + 1] = #10)) or
    ((Buf[Run] > #13) and (Buf[Run] <= #32)))
    do inc(Run);
end;

procedure TmwD4NLexer.SquareCloseHandler;
begin
  inc(Run);
  Id := leSquareClose;
end;

procedure TmwD4NLexer.SquareOpenHandler;
begin
  inc(Run);
  Id := leSquareOpen;
end;

procedure TmwD4NLexer.UnknownHandler;
begin
  Id := leUnknown;
  case Run + UTF8Width[Buf[Run]] < TheEnd of
    True: inc(Run, UTF8Width[Buf[Run]]);
    False: inc(Run);
  end;
end;

procedure TmwD4NLexer.UTF8IdentifierHandler;
begin
  inc(Run, UTF8Width[Buf[Run]]);
  Id := leIdentifier;
  ExId := leUTF8Identifier;
  while (Run < TheEnd) do
    case InInternationalIdentifiers[Buf[Run]] of
      True: inc(Run);
      False:
        case (Buf[Run] > #191) and (Run + UTF8Width[Buf[Run]] <= TheEnd) of
          True: inc(Run, UTF8Width[Buf[Run]]);
          False: Break;
        end;
    end;
end;

end.